# -*- coding: utf-8 -*-

import glob
import codecs
import string
import re
import d15n_routines as r
from dm_surname_cz import *

# create dictionary of w2m surname mapping from file
w2m_from_file = w2m_file()

# read lists of man and woman names
cz_man_names = r.cz_man_names()
cz_woman_names = r.cz_woman_names()

# binary end of line to be used
n = '\x0D\x0A'
""" 
    Data unification and cleanup routines

    Last update: 2011-06-07 Presovsky, Viktor

    a) create plain ascii uppercase version of data
    b) identify (repair if possible) first level of errors:
        ba) titles in name or surname (if found remove)
            e.g. ING.; MUDR.; PROF.; CSC;
        bb) unknown (probably misspeled or nonsense) names
            e.g. ADAMAEVA; MANZELE; V.
        bc) empty values (will be marked as <NULL>)
"""
tl = open('#cleanup_names_too_long_names.dat', 'wb')
ts = open('#cleanup_names_too_short_names.dat', 'wb')


d = open('#cleanup_names_dropped_lines.dat', 'wb')
t = open('#cleanup_names_titles_replaced.dat', 'wb')

counters = {'cnt_all_lines':0, 'cnt_titles_repair':0}

#for f in glob.glob('data_part1.txt'):
for f in glob.glob('POC_ID_firstname_surname_decline_v3.csv'):
    print f
    i = open(f, 'rb')
    o = open('_cleanup_' + f[:-4] + '.csv', 'wb')

    counters['cnt_all_lines'] = 0
    for l in i.readlines():

        counters['cnt_all_lines'] += 1
        l = r.remove_eol(l)

        #print l
        if len(l) == 0:
            # skip empty rows
            d.write(f + ' -- ' + l + n)
        elif counters['cnt_all_lines'] == 1: # skip headers
            d.write(f + ' -- ' + l + n)
        else:
            # remove multiple spaces, comas
            l = re.sub('[ ,]', '', l)
            l = r.ascii_upper(l)
            cols = string.split(l, '|')
            
            id = cols[0]; nm = cols[1]; sm = cols[2]
            id = id.strip(); nm = nm.strip(); sm = sm.strip()

            # report short/long names/surnames
            if len(nm) > 20 or len(sm) > 20: # too long sur/names
                tl.write(f + ' -- ' + l + n)
            if len(nm) < 3 or len(sm) < 3: # too short sur/names
                ts.write(f + ' -- ' + l + n)

            if nm in ('NONE', ''):
                nm = '<EMPTY>'
                print '<EMPTY>',
            if sm in ('NONE', ''):
                sm = '<EMPTY>'
                print '<EMPTY>',

            res = "%s\t%s\t%s" % (id, nm, sm)

            res2 = r.remove_titles(res)

            if res2 <> res:
                t.write(f + '\t>\t' + res + '\t>\t' + res2 + n)
                counters['cnt_titles_repair'] += 1
                res = res2[:]
            
            if nm in cz_man_names:
                app = "\t%s\t%s" % ('', sm)
            elif nm in cz_woman_names: 
                if w2m_from_file.has_key(sm):
                    m_sm = w2m_from_file[sm]
                else:
                    m_sm = w2m_generic(sm)
                app = "\t%s\t%s" % ('w', m_sm)
            else:
                app = "\t%s\t%s" % ('', '')
            
            o.write(res + app + n)
    i.close(); o.close()
d.close(); t.close()
tl.close(); ts.close()

print "counters['cnt_all_lines']", str(counters['cnt_all_lines'])
print "counters['cnt_titles_repair']", str(counters['cnt_titles_repair'])
